# ---------------爬取亚马逊---------------------------
# import requests
# url="https://www.amazon.cn/dp/B071GB8TYZ/ref=cngwdyfloorv2_recs_0/461-5685804-0539751?pf_rd_m=A1AJ19PSB66TGU&pf_rd_s=desktop-2&pf_rd_r=2MNFYGCQQABHDPRC8PHX&pf_rd_r=2MNFYGCQQABHDPRC8PHX&pf_rd_t=36701&pf_rd_p=538883d8-3c6e-4f6f-89c9-fedbbcdc164f&pf_rd_p=538883d8-3c6e-4f6f-89c9-fedbbcdc164f&pf_rd_i=desktop"
#
# try:
#     r =requests.get(url)
#     r.raise_for_status()
#     r.status_code
#     r.encoding
#     print(r.request.headers)
#     # user-Agent 'User-Agent': 'python-requests/2.19.1
#     print(r.json())
#
# except:
#     print("爬取失败")
# -------------更改headers-------------------
#
# import requests
#
#
# url = "https://www.amazon.cn/dp/B072J59473/ref=gbph_img_d-3_f4ba_ac9b1e34?smid=A1AJ19PSB66TGU&pf_rd_p=5254d6d3-aeff-45ee-8643-c4addbecf4ba&pf_rd_s=desktop-3&pf_rd_t=36701&pf_rd_i=desktop&pf_rd_m=A1AJ19PSB66TGU&pf_rd_r=2MNFYGCQQABHDPRC8PHX"
#
# try:
#     kv = {'user-agent': 'Mozilla/5.0'}
#     r = requests.get(url,headers=kv)
#     r.raise_for_status()
#     r.status_code
#     r.encoding=r.apparent_encoding
#     print(r.request.headers)
#     print(r.json())
#
# except:
#     print("爬取失败")
# ---------------百度关键词提交关键词------------------------
# import requests
# kv={'wd':'python'}
#
# try:
#     r =requests.get("http://www.baidu.com/s",params=kv)
#     r.raise_for_status()
#     print(r.status_code)
#     print(r.request.url)
#     print(len(r.text))
# except:
#     print("发送关键词失败")
#

# ------------------360 关键词提交关键词搜索---------------------
# import requests
# kv={'q':'python'}
# try:
#     r =requests.get("http://www.so.com/s",params=kv)
#     r.raise_for_status()
#     print(r.status_code)
#     print(r.request.url)
#     print(len(r.text))
# except:
#     print("发送关键词失败")
# ---------------百度图片的爬取---------------------
import urllib,os,re,requests
import json
def get_response(url):
    try:
        kv = {'user-agent': 'Mozilla/5.0'}
        response = urllib.requests.urlopen(url,kv)
        html=response.read()
        print(html)
        return html
    except:
        print("连接失败")
def get_content(html):
    # html=html.decode("utf8")
    reg = re.compile(r'(<body class="cms-area-hidden"">.*?</div>*?</body>)', re.S)
    print(re.findall(reg,html))
    return re.findall(reg,html) # 返回Picurl的列表

def main(url):
    get_content(get_response(url))

if __name__ == '__main__':
    url="http://www.baidu.com/s"
    main(url)